﻿using HtmlAgilityPack;
using System.Linq;
using System.Net;
using System.Net.Http;
using Newtonsoft.Json;
using System.IO;
using System;
using System.Threading.Tasks;
using System.Text.RegularExpressions;

namespace Moven.Extractor
{
    public class Program
    {
        const int StartArticleIndex = 2, EndArticleIndex = 2;
        const string BaseUrlPrefix = "http://cdn.moji.com/page5/page5/html/page-",
            BaseUrlSuffix = ".html",
            JsonFilePath = "articles.json",
            CompressedJsonFilePath = "compressed_articles.json",
            UpdateJsonFilePath = "updated_articles.json",
            CompressedUpdateJsonFilePath = "compressed_updated_articles.json";

        static readonly Regex ImgTagUrlPattern = new Regex(
            @"cdn\.moji\.com\/page5\/images\/uploads\/editor\/([0-9]{8})");

        class Article
        {
            public int Index { get; set; }
            public string Url { get; set; }
            public string Title { get; set; }
            public string[] Tags { get; set; }
            public string Date { get; set; }
        }

        class MovenArticles
        {
            public DateTime? UpdateTime;
            public int MaxIndex;
            public Article[] Articles;
        }
        public static void Main()
        {
            //BuildArticlesUpdate(1690, 100).Wait();
            //WriteArticlsToCompressedJson(ReadMovenArticlesFromJson(UpdateJsonFilePath), CompressedUpdateJsonFilePath);
            //CompressJsonData();
            //RefreshArticleDates();
            var i = 240;
            Console.WriteLine($"i is ${i}");
			Console.Read();
        }

        static void GetNewArticles()
        {
            var articlesTask = RetrieveArticles(StartArticleIndex, EndArticleIndex);
            var movenArticles = ReadMovenArticlesFromJson();
            var newArticles = articlesTask.Result;
            movenArticles.Articles = newArticles.Concat(movenArticles.Articles).
                ToArray();
            foreach (var article in movenArticles.Articles)
            {
                var title = article.Title;
                var parts = title.Split(' ');
                if (parts.Length > 1)
                {
                    title = parts[1];
                }
                var subtitles = title.Split('_');
                var maxLength = subtitles.Max(x => x.Length);
                article.Title = subtitles.First(x => x.Length == maxLength);
            }

            movenArticles.UpdateTime = DateTime.UtcNow;
            movenArticles.MaxIndex = movenArticles.Articles.Max(x => x.Index);
            var json = JsonConvert.SerializeObject(movenArticles, Formatting.Indented);
            File.WriteAllText(JsonFilePath, json);
        }

        static MovenArticles ReadMovenArticlesFromJson(string path = JsonFilePath) => JsonConvert.DeserializeObject<MovenArticles>(File.ReadAllText(path));

        static void RefreshArticleDates()
        {
            var movenArticles = ReadMovenArticlesFromJson();
            RefreshArticleDates(movenArticles.Articles);
            var json = JsonConvert.SerializeObject(movenArticles, Formatting.Indented);
            File.WriteAllText(JsonFilePath, json);
            WriteArticlsToCompressedJson(movenArticles);
        }

        static string GetArticleDate(string html)
        {
            if (string.IsNullOrEmpty(html))
            {
                return string.Empty;
            }
            var match = ImgTagUrlPattern.Match(html);
            if (!match.Success)
            {
                return string.Empty;
            }
            var dateGroup = match.Groups[1];
            var date = dateGroup.Value;
            return $"{date.Substring(0, 4)}-{date.Substring(4, 2)}-{date.Substring(6)}";
        }

        static void RefreshArticleDate(Article article)
        {
            if (!string.IsNullOrEmpty(article.Date))
            {
                return;
            }
            try
            {
                var result = RetrieveArticle(article.Url).Result;
                var html = result.Item1;
                var date = result.Item2 ?? GetArticleDate(html);
                article.Date = date;
            }
            catch
            {
            }
        }
        
        static void RefreshArticleDates(Article[] articles)
        {
            Console.WriteLine($"Refreshing the publication dates of {articles.Length} articles");
            //RefreshArticleDate(articles[0]);
            articles.AsParallel().ForAll(RefreshArticleDate);
        }

        static void CompressJsonData() => WriteArticlsToCompressedJson(ReadMovenArticlesFromJson());
        
        static void WriteArticlsToCompressedJson(MovenArticles articles, string path = CompressedJsonFilePath)
        {
            var json = JsonConvert.SerializeObject(articles, Formatting.None);
            File.WriteAllText(path, json);
        }

        static string IndexToUrl(int index) => BaseUrlPrefix + index + BaseUrlSuffix;

        static async Task BuildArticlesUpdate(int maxIndex, int offset = 200)
        {
            var articles = await RetrieveArticles(maxIndex + 1, maxIndex + 1 + offset);
            if (!articles.Any())
            {
                return;
            }
            var movenArticles = new MovenArticles
            {
                Articles = articles,
                MaxIndex = articles.Max(x => x.Index),
                UpdateTime = DateTime.UtcNow.Date
            };
            var json = JsonConvert.SerializeObject(articles, Formatting.Indented);

            File.WriteAllText(UpdateJsonFilePath, json);
        }

        static async Task<Tuple<string, string>> RetrieveArticle(string url)
        {
            using (var httpClient = new HttpClient())
            {
                var response = await httpClient.GetAsync(url);
                var content = await response.Content.ReadAsStringAsync();
                var lastModified = response.Content.Headers.LastModified;
                var formattedDate = lastModified?.ToString("yyyy-MM-dd");
                return Tuple.Create(content, formattedDate);
            }
        }

        static async Task<Article[]> RetrieveArticles(int startIndex, int endIndex)
        {
            var indices = Enumerable.Range(startIndex, endIndex - startIndex + 1);
            var tasks = indices.AsParallel().AsOrdered().Select(
                async index =>
                {
                    var url = IndexToUrl(index);
                    try
                    {
                        Console.WriteLine("Retrieving article with index " + index);
                        var result = await RetrieveArticle(url);
                        var html = result.Item1;
                        var date = result.Item2;
                        if (string.IsNullOrEmpty(html))
                        {
                            return null;
                        }
                        html = WebUtility.HtmlDecode(html);
                        var htmlDoc = new HtmlDocument();
                        htmlDoc.LoadHtml(html);
                        var titleElement = htmlDoc.DocumentNode.Element("html")
                                .Element("head").Element("title");
                        if (titleElement == null)
                        {
                            return null;
                        }
                        var title = titleElement.InnerText;
                        Console.WriteLine("Retrieved article with index " + index + " with title " + title);
                        var badTitleKeywords = new []{"test", "测试", "错误"};
                        return badTitleKeywords.Any(word => title.StartsWith(word, StringComparison.OrdinalIgnoreCase))? null : new Article
                        {
                            Index = index,
                            Title = title,
                            Url = url,
                            Tags = new string[0],
                            Date = date?? GetArticleDate(html)
                        };
                    }
                    catch(Exception ex)
                    {
                        Console.WriteLine("Error retrieving article with index " + index + ex);
                        return null;
                    }
                }
            ).ToArray();
            await Task.WhenAll(tasks);

            return (from task in tasks
                    where task.IsCompleted && task.Result != null
                    select task.Result).ToArray();
        }
    }
}
